
# imports
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
import datetime
from sklearn.linear_model import LinearRegression
import warnings
warnings.filterwarnings('ignore')


#Read the csv files
yr17_18_data = pd.read_csv("17_18_data.csv")
yr20_21_data = pd.read_csv("20_21_data.csv")


#Observe the 2017-2018 data
yr17_18_data.head()


#Observe the 2020-2021 data
yr20_21_data.head()


# I'm going to make a new table in the correct order so it will be easier to combine with the 2017-2018 data

total_data = pd.DataFrame(columns = ['video_id', 'title', 'publish_time', 'channel_title', 'category_id', 'trending_date', 'tags', 'view_count', 'likes', 'dislikes', 'comment_count', 'comments_disabled', 'ratings_disabled' ,'description'])

#Converted the trending_date data from Series to 
yr20_21_data['publishedAt'] = pd.to_datetime(yr20_21_data['publishedAt'], ).dt.tz_localize(None)
yr17_18_data['publish_time'] = pd.to_datetime(yr17_18_data['publish_time']).dt.tz_localize(None)
yr20_21_data['trending_date'] = pd.to_datetime(yr20_21_data['trending_date']).dt.tz_localize(None)
yr17_18_data['trending_date'] = pd.to_datetime(yr17_18_data['trending_date'], format='%y.%d.%m').dt.tz_localize(None)


total_data['video_id'] = yr20_21_data['video_id'].append(yr17_18_data['video_id'])
total_data['trending_date'] = yr20_21_data['trending_date'].append(yr17_18_data['trending_date'])
total_data['title'] = yr20_21_data['title'].append(yr17_18_data['title'])
total_data['channel_title'] = yr20_21_data['channelTitle'].append(yr17_18_data['channel_title'])
total_data['category_id'] = yr20_21_data['categoryId'].append(yr17_18_data['category_id'])
total_data['publish_time'] = yr20_21_data['publishedAt'].append(yr17_18_data['publish_time'])
total_data['tags'] = yr20_21_data['tags'].append(yr17_18_data['tags'])
total_data['view_count'] = yr20_21_data['view_count'].append(yr17_18_data['views'])
total_data['likes'] = yr20_21_data['likes'].append(yr17_18_data['likes'])
total_data['dislikes'] = yr20_21_data['dislikes'].append(yr17_18_data['dislikes'])
total_data['comment_count'] = yr20_21_data['comment_count'].append(yr17_18_data['comment_count'])
total_data['comments_disabled'] = yr20_21_data['comments_disabled'].append(yr17_18_data['comments_disabled'])
total_data['ratings_disabled'] = yr20_21_data['ratings_disabled'].append(yr17_18_data['ratings_disabled'])
total_data['description'] = yr20_21_data['description'].append(yr17_18_data['description'])


total_data = total_data.drop_duplicates(subset = ['video_id','title'], keep = "first")
total_data['publish_time']

0       2020-08-11 19:20:14
1       2020-08-11 17:00:10
2       2020-08-11 16:34:06
3       2020-08-11 16:38:55
4       2020-08-11 15:10:05
                ...        
40760   2018-06-12 18:55:26
40761   2018-06-13 15:06:08
40762   2018-06-13 05:27:27
40764   2018-06-12 16:03:58
40766   2018-06-13 09:00:06
Name: publish_time, Length: 16090, dtype: datetime64[ns]


total_data.isnull().sum()

video_id               0
title                  0
publish_time           0
channel_title          0
category_id            0
trending_date          0
tags                   0
view_count             0
likes                  0
dislikes               0
comment_count          0
comments_disabled      0
ratings_disabled       0
description          184
dtype: int64


total_data['description'] = total_data['description'].fillna(value = "N/A")


total_data.describe(include = ['O'])


total_data.head()


graph = plt.subplots(figsize=(10,6))
corr_map = sns.heatmap(total_data.corr(), annot=True, cmap="viridis")


#grouped data by channel title and counted up the number of times appeared on Youtube trending
trending_data_count = total_data.groupby(by = 'channel_title').agg('count')
#sorted from top to bottom
trending_data_count = trending_data_count.sort_values(by = ['video_id'], ascending=False).iloc[:10]
#reset the index
trending_data_count.reset_index(level=0, inplace=True)
#created the plot
g = trending_data_count['video_id'].plot(figsize = (15,15), kind ="barh")
labels = trending_data_count['channel_title']
g.set_yticklabels(labels, fontsize = 10)
plt.title('Top 50 trending channels from 2017-2021 (Excluding 2019 data)', fontsize = 10)
# trending_data_count.plot(figsize = (100,20), rot = 1)

Text(0.5, 1.0, 'Top 50 trending channels from 2017-2021 (Excluding 2019 data)')


##created a function to plot graphs based on the parameters
def top_data (data, type, b, num):
    #sorts and removed duplicates
    view_count_sorted_tb = data.sort_values(by = type, ascending= b ).iloc[:num].drop_duplicates("title",keep="first")
    #get the labels
    channel_titles = view_count_sorted_tb['title']
    ax1 = view_count_sorted_tb[type].plot(figsize = (15,15), kind ="barh")
    
    ax2 = view_count_sorted_tb[['likes', 'dislikes', 'comment_count']].plot(figsize = (10,10), kind ="barh")
    ax1.set_yticklabels(channel_titles, fontsize = 10)
    ax2.set_yticklabels(channel_titles, fontsize = 10)


top_data(total_data, ['view_count'], False, 10)


top_data(total_data, ['likes'], False, 10)


top_data(total_data, ['dislikes'], False, 10)


top_data(total_data, ['view_count'], True, 10)


import matplotlib.pyplot as plt
# I sorted the values based on view count for the top 50 videos as my sample
view_count_categories = total_data.sort_values(by = ['view_count'], ascending=False).iloc[:50]
# I aggregated the data sorting by category
view_count_categories = view_count_categories.groupby(by = 'category_id').agg('count')
view_count_categories.reset_index(level=0, inplace=True)
#plotted the pie chart
fig1, ax1 = plt.subplots()
ax1.pie(view_count_categories['video_id'], labels = view_count_categories['category_id'], autopct='%1.1f%%',
        shadow=True, startangle=90)
ax1.axis('equal') 
ax1.set_title("Most viewed category ids")
plt.show()


top_data(yr20_21_data, ['view_count'], False, 40)


top_data(yr17_18_data, ['views'], False, 40)


from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
from matplotlib import *
def word_clouder(tb,type):
    top_views = tb.sort_values(by = [type], ascending=False).iloc[:40]
    top_tags = []
    for col, row in tb.iterrows():
        vid_tags = row['tags'].split('|')
        for j in vid_tags:
            top_tags.append(j)
    stopwords = STOPWORDS.add("None")
    stopwords = STOPWORDS.add("make")
    stopwords = STOPWORDS.add("girl")
    stopwords = STOPWORDS.add("new")
    stopwords = STOPWORDS.add("first take")
    stopwords = STOPWORDS.add("highlight")

    unique_string=(" ").join(top_tags)
    wordcloud = WordCloud(width = 1000, height = 1000, stopwords  = stopwords).generate(unique_string)
    plt.figure(figsize=(10,10))
    plt.imshow(wordcloud)   
    plt.axis("off")
    plt.subplot()
    plt.show()


word_clouder(total_data, 'tags')


word_clouder(yr17_18_data, 'tags')


word_clouder(yr20_21_data, 'tags')


ratings_disabled_tb = yr20_21_data.loc[yr20_21_data['ratings_disabled'] == True]
stopwords = STOPWORDS.add("None")
stopwords = STOPWORDS.add("make")
stopwords = STOPWORDS.add("girl")
stopwords = STOPWORDS.add("new")
stopwords = STOPWORDS.add("ASKED")
stopwords = STOPWORDS.add("GIRLFRIEND")
stopwords = STOPWORDS.add("brings")
stopwords = STOPWORDS.add("FIRST")
stopwords = STOPWORDS.add("Day")

wordcloud = WordCloud(
                          background_color='black',
                          stopwords=stopwords,
                          max_words=1000,
                          max_font_size=120, 
                          random_state=40
                         ).generate(str(ratings_disabled_tb['title']))

print(wordcloud)
fig = plt.figure(1)
plt.imshow(wordcloud)
plt.title("WORD CLOUD")
plt.axis('off')
plt.show()

<wordcloud.wordcloud.WordCloud object at 0x000001C44216F790>


comments_disabled_tb = yr20_21_data[yr20_21_data['comments_disabled'] == True]

stopwords = STOPWORDS.add("None")
stopwords = STOPWORDS.add("make")
stopwords = STOPWORDS.add("girl")
stopwords = STOPWORDS.add("new")
stopwords = STOPWORDS.add("ASKED")
stopwords = STOPWORDS.add("GIRLFRIEND")
stopwords = STOPWORDS.add("brings")
stopwords = STOPWORDS.add("FIRST")
stopwords = STOPWORDS.add("Day")
wordcloud = WordCloud(
                          background_color='black',
                          stopwords=stopwords,
                          max_words=1000,
                          max_font_size=120, 
                          random_state=40
                         ).generate(str(comments_disabled_tb['title']))

print(wordcloud)
fig = plt.figure(1)
plt.imshow(wordcloud)
plt.title("WORD CLOUD")
plt.axis('off')
plt.show()

<wordcloud.wordcloud.WordCloud object at 0x000001C421B8BE50>


ml_data = total_data.drop(['video_id', 'publish_time', 'description'], axis = 1)
ml_data['tags'] = ml_data['tags'].replace('|',' ')
# converted boolean to integer
ml_data['comments_disabled'] = ml_data['comments_disabled'].astype(int)
ml_data['ratings_disabled'] = ml_data['ratings_disabled'].astype(int)
ml_data.head()


# Made a new list based on categories
categories_in_order = total_data['category_id']


#converted the years into int64 from datetime64[ns]
ml_data['years'] = ml_data['trending_date'].dt.year
ml_data['years'].head()

0    2020
1    2020
2    2020
3    2020
4    2020
Name: years, dtype: int64


categories = ml_data['category_id'].unique()
category_data = []
for i in categories:
    category_data = ml_data[ml_data["category_id"] == i]
    #Remove unnecessary data columns
    category_data = category_data.drop(columns = ['title', 'channel_title', 'trending_date', 'category_id', 'tags', 'likes', 'dislikes', 'comment_count', 'comments_disabled', 'ratings_disabled', 'trending_date' ])
    #get the correct data
    x_value = category_data.iloc[:, 1].values.reshape(-1,1)
    y = category_data.iloc[:, :-1].values.reshape(-1,1)

    #train the linear regression model
    x_trainer, x_test, y_trainer, y_test = train_test_split(x_value, y, test_size=0.2, random_state=0)

    linRegress = LinearRegression()
    linRegress.fit(x_trainer, y_trainer)
    print("Coefficient: %.9f" % linRegress.coef_)
    print("Squared Err: %.2f" % np.mean((linRegress.predict(x_test) - y_test) ** 2))
    print('Variance: %.2f' % linRegress.score(x_test, y_test))
    category_data.plot.scatter(x='years', y = 'view_count')

    plt.title(i)
    plt.plot(x_value, linRegress.predict(x_value), color='k')
    plt.show()

Coefficient: 161647.399677890
Squared Err: 2865708019244.33
Variance: 0.02

Coefficient: -97931.400404449
Squared Err: 2701594838705.62
Variance: -0.02

Coefficient: 180819.682448685
Squared Err: 3973760692755.90
Variance: 0.03

Coefficient: 36728.209920488
Squared Err: 10894195685231.83
Variance: 0.00

Coefficient: 122960.689212842
Squared Err: 535757504248.79
Variance: -0.02

Coefficient: 131882.438122442
Squared Err: 336735736325.52
Variance: 0.09

Coefficient: 19083.135470396
Squared Err: 1123567330259.09
Variance: -0.00

Coefficient: 384787.127667056
Squared Err: 4043206195322.36
Variance: 0.03

Coefficient: 109023.843916868
Squared Err: 2602971598031.57
Variance: -0.01

Coefficient: 303780.613883912
Squared Err: 567489852316.55
Variance: -0.02


from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
data_cp = pd.DataFrame(columns=['years', 'likes'])
data_cp['years'] = ml_data['years']
data_cp['likes'] = ml_data ['likes']
# data_cp
#get the values
x_value = data_cp.iloc[:, :-1].values
y = data_cp.iloc[:, 1].values
#train the values
X_train, X_test, y_train, y_test = train_test_split(x_value, y, test_size=0.2, random_state=0)
#do a linear regression
linRegress = LinearRegression()
#Fit it to the graph
linRegress.fit(X_train, y_train)
print("Coefficient: %.9f" % linRegress.coef_)
print("Intercept: %.9f" % linRegress.intercept_)
print("Squared Err: %.2f" % np.mean((linRegress.predict(x_test) - y_test) ** 2))
print('Variance: %.2f' % linRegress.score(x_test, y_test))

#make a scatter plot
ml_data.plot.scatter(x='years', y = 'likes')
#add the linear regression line
plt.plot(x_value, linRegress.predict(x_value), color='k')
plt.show()

Coefficient: 20509.911346385
Intercept: -41342620.832740121
Squared Err: 35268308750.56
Variance: 0.02


ml_data['years'].unique
sns.violinplot(x = 'years', y = 'likes', data = ml_data, bw =.3 )
plt.title("Violin plot of likes vs year")
plt.show()


I also plotted the residuals of the likes to years


ml_data['years'].unique
#make a violin plot
ml_data['residual'] = ml_data['likes'] - (linRegress.intercept_ + linRegress.coef_[0]*ml_data['years'])

sns.violinplot(x = 'years', y = 'residual', data = ml_data, bw =.3 )
plt.title("Violin plot of residuals vs year")
plt.show()

Category ID	Subject
1	Film & Animation
2	Autos & Vehicles
10	Music
15	Pets & Animals
17	Sports
18	Short Movies
19	Travel & Events
20	Gaming
21	Videoblogging
22	People & Blogs
23	Comedy
24	Entertainment
25	News & Politics
26	Howto & Style
27	Education
28	Science & Technology
29	Nonprofits & Activism
30	Movies
31	Anime/Animation
32	Action/Adventure
33	Classics
34	Comedy
35	Documentary
36	Drama
37	Family
38	Foreign
39	Horror
40	Sci-Fi/Fantasy
41	Thriller
42	Shorts
44	Trailers

	video_id	trending_date	title	channel_title	category_id	publish_time	tags	views	likes	dislikes	comment_count	thumbnail_link	comments_disabled	ratings_disabled	video_error_or_removed	description
0	2kyS6SvSYSE	17.14.11	WE WANT TO TALK ABOUT OUR MARRIAGE	CaseyNeistat	22	2017-11-13T17:13:01.000Z	SHANtell martin	748374	57527	2966	15954	https://i.ytimg.com/vi/2kyS6SvSYSE/default.jpg	False	False	False	SHANTELL'S CHANNEL - https://www.youtube.com/s...
1	1ZAPwfrtAFY	17.14.11	The Trump Presidency: Last Week Tonight with J...	LastWeekTonight	24	2017-11-13T07:30:00.000Z	last week tonight trump presidency\|"last week ...	2418783	97185	6146	12703	https://i.ytimg.com/vi/1ZAPwfrtAFY/default.jpg	False	False	False	One year after the presidential election, John...
2	5qpjK5DgCt4	17.14.11	Racist Superman \| Rudy Mancuso, King Bach & Le...	Rudy Mancuso	23	2017-11-12T19:05:24.000Z	racist superman\|"rudy"\|"mancuso"\|"king"\|"bach"...	3191434	146033	5339	8181	https://i.ytimg.com/vi/5qpjK5DgCt4/default.jpg	False	False	False	WATCH MY PREVIOUS VIDEO ▶ \n\nSUBSCRIBE ► http...
3	puqaWrEC7tY	17.14.11	Nickelback Lyrics: Real or Fake?	Good Mythical Morning	24	2017-11-13T11:00:04.000Z	rhett and link\|"gmm"\|"good mythical morning"\|"...	343168	10172	666	2146	https://i.ytimg.com/vi/puqaWrEC7tY/default.jpg	False	False	False	Today we find out if Link is a Nickelback amat...
4	d380meD0W0M	17.14.11	I Dare You: GOING BALD!?	nigahiga	24	2017-11-12T18:01:41.000Z	ryan\|"higa"\|"higatv"\|"nigahiga"\|"i dare you"\|"...	2095731	132235	1989	17518	https://i.ytimg.com/vi/d380meD0W0M/default.jpg	False	False	False	I know it's been a while since we did this sho...

	video_id	title	publishedAt	channelId	channelTitle	categoryId	trending_date	tags	view_count	likes	dislikes	comment_count	thumbnail_link	comments_disabled	ratings_disabled	description
0	3C66w5Z0ixs	I ASKED HER TO BE MY GIRLFRIEND...	2020-08-11T19:20:14Z	UCvtRTOMP2TqYqu51xNrqAzg	Brawadis	22	2020-08-12T00:00:00Z	brawadis\|prank\|basketball\|skits\|ghost\|funny vi...	1514614	156908	5855	35313	https://i.ytimg.com/vi/3C66w5Z0ixs/default.jpg	False	False	SUBSCRIBE to BRAWADIS ▶ http://bit.ly/Subscrib...
1	M9Pmf9AB4Mo	Apex Legends \| Stories from the Outlands – “Th...	2020-08-11T17:00:10Z	UC0ZV6M2THA81QT9hrVWJG3A	Apex Legends	20	2020-08-12T00:00:00Z	Apex Legends\|Apex Legends characters\|new Apex ...	2381688	146739	2794	16549	https://i.ytimg.com/vi/M9Pmf9AB4Mo/default.jpg	False	False	While running her own modding shop, Ramya Pare...
2	J78aPJ3VyNs	I left youtube for a month and THIS is what ha...	2020-08-11T16:34:06Z	UCYzPXprvl5Y-Sf0g4vX-m6g	jacksepticeye	24	2020-08-12T00:00:00Z	jacksepticeye\|funny\|funny meme\|memes\|jacksepti...	2038853	353787	2628	40221	https://i.ytimg.com/vi/J78aPJ3VyNs/default.jpg	False	False	I left youtube for a month and this is what ha...
3	kXLn3HkpjaA	XXL 2020 Freshman Class Revealed - Official An...	2020-08-11T16:38:55Z	UCbg_UMjlHJg_19SZckaKajg	XXL	10	2020-08-12T00:00:00Z	xxl freshman\|xxl freshmen\|2020 xxl freshman\|20...	496771	23251	1856	7647	https://i.ytimg.com/vi/kXLn3HkpjaA/default.jpg	False	False	Subscribe to XXL → http://bit.ly/subscribe-xxl...
4	VIUo6yapDbc	Ultimate DIY Home Movie Theater for The LaBran...	2020-08-11T15:10:05Z	UCDVPcEbVLQgLZX0Rt6jo34A	Mr. Kate	26	2020-08-12T00:00:00Z	The LaBrant Family\|DIY\|Interior Design\|Makeove...	1123889	45802	964	2196	https://i.ytimg.com/vi/VIUo6yapDbc/default.jpg	False	False	Transforming The LaBrant Family's empty white ...

	video_id	title	publish_time	channel_title	category_id	trending_date	tags	view_count	likes	dislikes	comment_count	comments_disabled	ratings_disabled	description
0	3C66w5Z0ixs	I ASKED HER TO BE MY GIRLFRIEND...	2020-08-11 19:20:14	Brawadis	22	2020-08-12	brawadis\|prank\|basketball\|skits\|ghost\|funny vi...	1514614	156908	5855	35313	False	False	SUBSCRIBE to BRAWADIS ▶ http://bit.ly/Subscrib...
1	M9Pmf9AB4Mo	Apex Legends \| Stories from the Outlands – “Th...	2020-08-11 17:00:10	Apex Legends	20	2020-08-12	Apex Legends\|Apex Legends characters\|new Apex ...	2381688	146739	2794	16549	False	False	While running her own modding shop, Ramya Pare...
2	J78aPJ3VyNs	I left youtube for a month and THIS is what ha...	2020-08-11 16:34:06	jacksepticeye	24	2020-08-12	jacksepticeye\|funny\|funny meme\|memes\|jacksepti...	2038853	353787	2628	40221	False	False	I left youtube for a month and this is what ha...
3	kXLn3HkpjaA	XXL 2020 Freshman Class Revealed - Official An...	2020-08-11 16:38:55	XXL	10	2020-08-12	xxl freshman\|xxl freshmen\|2020 xxl freshman\|20...	496771	23251	1856	7647	False	False	Subscribe to XXL → http://bit.ly/subscribe-xxl...
4	VIUo6yapDbc	Ultimate DIY Home Movie Theater for The LaBran...	2020-08-11 15:10:05	Mr. Kate	26	2020-08-12	The LaBrant Family\|DIY\|Interior Design\|Makeove...	1123889	45802	964	2196	False	False	Transforming The LaBrant Family's empty white ...

Category ID	Subject
1	Film & Animation
2	Autos & Vehicles
10	Music
15	Pets & Animals
17	Sports
18	Short Movies
19	Travel & Events
20	Gaming
21	Videoblogging
22	People & Blogs
23	Comedy
24	Entertainment
25	News & Politics
26	Howto & Style
27	Education
28	Science & Technology
29	Nonprofits & Activism
30	Movies
31	Anime/Animation
32	Action/Adventure
33	Classics
34	Comedy
35	Documentary
36	Drama
37	Family
38	Foreign
39	Horror
40	Sci-Fi/Fantasy
41	Thriller
42	Shorts
44	Trailers

Youtube Data Analysis and 2019 Data Predictions¶

CMSC320¶

By Rachel Taskale¶

About The Data:¶

Cleaning the Data:¶

EDA¶

Part 4: Linear Regression Model¶

Conclusion¶

	video_id	title	channel_title	tags	description
count	16090	16090	16090	16090	16090
unique	15730	16037	4802	13229	15459
top	WQ8Xgp8ALFo	Starlink Mission	NBA	[None]	N/A
freq	6	15	165	1255	184